# scratch 大时代1994-柳岸话又明 拉取目录

import urllib.request
from bs4 import BeautifulSoup
import ssl
import time

# Ignore SSL certificate errors
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE

host = "https://m.boquge.com"

# Ask for URL input
url = input('Enter a url address (default https://m.boquge.com/wapbook/90695-1.html): ')
if url == '':
    url = 'https://m.boquge.com/wapbook/90695-1.html'

# Set up headers to mimic a real browser
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'}

# generates all index-address.
all_index_address = [
    'https://m.boquge.com/wapbook/90695-1.html',
    'https://m.boquge.com/wapbook/90695-2.html',
    'https://m.boquge.com/wapbook/90695-3.html',
    'https://m.boquge.com/wapbook/90695-4.html',
    'https://m.boquge.com/wapbook/90695-5.html',
    'https://m.boquge.com/wapbook/90695-6.html',
    'https://m.boquge.com/wapbook/90695-7.html',
    'https://m.boquge.com/wapbook/90695-8.html',
    'https://m.boquge.com/wapbook/90695-9.html',
    'https://m.boquge.com/wapbook/90695-10.html',

    'https://m.boquge.com/wapbook/90695-11.html',
    'https://m.boquge.com/wapbook/90695-12.html',
    'https://m.boquge.com/wapbook/90695-13.html',
    'https://m.boquge.com/wapbook/90695-14.html',
    'https://m.boquge.com/wapbook/90695-15.html',
    'https://m.boquge.com/wapbook/90695-16.html',
    'https://m.boquge.com/wapbook/90695-17.html',
    'https://m.boquge.com/wapbook/90695-18.html',
    'https://m.boquge.com/wapbook/90695-19.html',
    'https://m.boquge.com/wapbook/90695-20.html',

    'https://m.boquge.com/wapbook/90695-21.html',
    'https://m.boquge.com/wapbook/90695-22.html',
    'https://m.boquge.com/wapbook/90695-23.html',
    'https://m.boquge.com/wapbook/90695-24.html',
    'https://m.boquge.com/wapbook/90695-25.html',
    'https://m.boquge.com/wapbook/90695-26.html',
    'https://m.boquge.com/wapbook/90695-27.html',
    'https://m.boquge.com/wapbook/90695-28.html',
    'https://m.boquge.com/wapbook/90695-29.html',
    'https://m.boquge.com/wapbook/90695-30.html',

    'https://m.boquge.com/wapbook/90695-31.html',
    'https://m.boquge.com/wapbook/90695-32.html',
    'https://m.boquge.com/wapbook/90695-33.html',
    'https://m.boquge.com/wapbook/90695-34.html',
]

for url in all_index_address:
    time.sleep(1)

    # Create a request object with headers
    req = urllib.request.Request(url, headers=headers)

    # Open the URL using the request object
    html = urllib.request.urlopen(req, context=ctx).read()

    # Parse the HTML with BeautifulSoup
    soup = BeautifulSoup(html, 'html.parser')

    content = []
    ul = soup.find('ul', id='chapterlist')
    lis = ul.find_all('li')
    for li in lis:
        a = li.find('a')
        localurl = a.get('href')
        # print(localurl)
        content.append(host + localurl)

    # print(title.getText())
    # print(content)

    # save the tile and the content to a file.
    filename = "大时代1994_柳岸花又明_index.txt"
    with open(filename, 'a', encoding='utf-8') as file:
        for line in content:
            file.write(line)
            file.write('\n')

